{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: nn模块\n",
    "-----------\n",
    "\n",
    "\n",
    "我们接下来使用nn模块来实现这个简单的全连接网络。前面我们通过用Tensor和Operation等low-level API来创建\n",
    "动态的计算图。这里我们使用更简单的high-level API。\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5.0a0+a24163a\n",
      "0 665.8291015625\n",
      "1 614.0447998046875\n",
      "2 569.7244262695312\n",
      "3 530.7206420898438\n",
      "4 495.89105224609375\n",
      "5 464.5169982910156\n",
      "6 436.1329650878906\n",
      "7 410.459228515625\n",
      "8 386.9529724121094\n",
      "9 365.2677307128906\n",
      "10 344.98291015625\n",
      "11 325.9330749511719\n",
      "12 308.0951843261719\n",
      "13 291.2874755859375\n",
      "14 275.3282165527344\n",
      "15 260.17352294921875\n",
      "16 245.74899291992188\n",
      "17 231.9853515625\n",
      "18 218.92092895507812\n",
      "19 206.46051025390625\n",
      "20 194.60479736328125\n",
      "21 183.24606323242188\n",
      "22 172.43678283691406\n",
      "23 162.18336486816406\n",
      "24 152.42430114746094\n",
      "25 143.16773986816406\n",
      "26 134.38656616210938\n",
      "27 126.10890197753906\n",
      "28 118.2713623046875\n",
      "29 110.8513412475586\n",
      "30 103.85079956054688\n",
      "31 97.25940704345703\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lili/py3.6-env/lib/python3.6/site-packages/torch/nn/functional.py:52: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32 91.06820678710938\n",
      "33 85.24884796142578\n",
      "34 79.78056335449219\n",
      "35 74.64293670654297\n",
      "36 69.823974609375\n",
      "37 65.31278991699219\n",
      "38 61.07954025268555\n",
      "39 57.120540618896484\n",
      "40 53.41852569580078\n",
      "41 49.95926284790039\n",
      "42 46.72844314575195\n",
      "43 43.7166748046875\n",
      "44 40.90652847290039\n",
      "45 38.291927337646484\n",
      "46 35.860286712646484\n",
      "47 33.591575622558594\n",
      "48 31.47673225402832\n",
      "49 29.512805938720703\n",
      "50 27.68549346923828\n",
      "51 25.98240852355957\n",
      "52 24.395750045776367\n",
      "53 22.916954040527344\n",
      "54 21.536808013916016\n",
      "55 20.245407104492188\n",
      "56 19.041471481323242\n",
      "57 17.920501708984375\n",
      "58 16.871746063232422\n",
      "59 15.892293930053711\n",
      "60 14.976529121398926\n",
      "61 14.120612144470215\n",
      "62 13.319745063781738\n",
      "63 12.570850372314453\n",
      "64 11.870187759399414\n",
      "65 11.21413516998291\n",
      "66 10.5996732711792\n",
      "67 10.024039268493652\n",
      "68 9.484028816223145\n",
      "69 8.977757453918457\n",
      "70 8.502097129821777\n",
      "71 8.053671836853027\n",
      "72 7.6325201988220215\n",
      "73 7.236522197723389\n",
      "74 6.864195346832275\n",
      "75 6.513209342956543\n",
      "76 6.182273864746094\n",
      "77 5.870550632476807\n",
      "78 5.576077461242676\n",
      "79 5.298644542694092\n",
      "80 5.037141799926758\n",
      "81 4.790079593658447\n",
      "82 4.556771755218506\n",
      "83 4.3362932205200195\n",
      "84 4.128203868865967\n",
      "85 3.9315402507781982\n",
      "86 3.745738983154297\n",
      "87 3.5699660778045654\n",
      "88 3.403337001800537\n",
      "89 3.245518922805786\n",
      "90 3.0959715843200684\n",
      "91 2.9541802406311035\n",
      "92 2.8196375370025635\n",
      "93 2.691917657852173\n",
      "94 2.5705902576446533\n",
      "95 2.4552407264709473\n",
      "96 2.3456380367279053\n",
      "97 2.241644859313965\n",
      "98 2.1428115367889404\n",
      "99 2.048957109451294\n",
      "100 1.9596436023712158\n",
      "101 1.8746753931045532\n",
      "102 1.7937204837799072\n",
      "103 1.7167245149612427\n",
      "104 1.643458604812622\n",
      "105 1.5735998153686523\n",
      "106 1.5070234537124634\n",
      "107 1.4435523748397827\n",
      "108 1.3830584287643433\n",
      "109 1.3252993822097778\n",
      "110 1.2701995372772217\n",
      "111 1.2175694704055786\n",
      "112 1.1673156023025513\n",
      "113 1.1192706823349\n",
      "114 1.0734062194824219\n",
      "115 1.0295517444610596\n",
      "116 0.9876620769500732\n",
      "117 0.9476345777511597\n",
      "118 0.9092988967895508\n",
      "119 0.8726337552070618\n",
      "120 0.8375756144523621\n",
      "121 0.8039571642875671\n",
      "122 0.771763801574707\n",
      "123 0.7409393191337585\n",
      "124 0.7114245891571045\n",
      "125 0.6831771731376648\n",
      "126 0.6561216115951538\n",
      "127 0.6302127838134766\n",
      "128 0.6053729057312012\n",
      "129 0.5815773606300354\n",
      "130 0.5587990283966064\n",
      "131 0.5369634032249451\n",
      "132 0.5160486102104187\n",
      "133 0.49606290459632874\n",
      "134 0.47690916061401367\n",
      "135 0.45852720737457275\n",
      "136 0.4408995509147644\n",
      "137 0.42398717999458313\n",
      "138 0.4077647924423218\n",
      "139 0.3922043442726135\n",
      "140 0.3773549199104309\n",
      "141 0.3630879521369934\n",
      "142 0.34939607977867126\n",
      "143 0.3362468481063843\n",
      "144 0.32361993193626404\n",
      "145 0.3115197718143463\n",
      "146 0.2999124228954315\n",
      "147 0.2887515127658844\n",
      "148 0.2780269980430603\n",
      "149 0.26772165298461914\n",
      "150 0.2578287720680237\n",
      "151 0.2483227401971817\n",
      "152 0.2391788810491562\n",
      "153 0.23039542138576508\n",
      "154 0.22195088863372803\n",
      "155 0.21382524073123932\n",
      "156 0.20600804686546326\n",
      "157 0.1984904408454895\n",
      "158 0.19126886129379272\n",
      "159 0.18431852757930756\n",
      "160 0.17763227224349976\n",
      "161 0.17119695246219635\n",
      "162 0.16500753164291382\n",
      "163 0.15906168520450592\n",
      "164 0.15333116054534912\n",
      "165 0.1478160172700882\n",
      "166 0.1425071507692337\n",
      "167 0.1373961716890335\n",
      "168 0.1324833631515503\n",
      "169 0.12774844467639923\n",
      "170 0.12319374084472656\n",
      "171 0.11880745738744736\n",
      "172 0.11458003520965576\n",
      "173 0.11050987988710403\n",
      "174 0.10659079253673553\n",
      "175 0.10281502455472946\n",
      "176 0.0991780012845993\n",
      "177 0.09567391127347946\n",
      "178 0.09230390191078186\n",
      "179 0.08906559646129608\n",
      "180 0.0859440490603447\n",
      "181 0.08293503522872925\n",
      "182 0.08003807812929153\n",
      "183 0.07724471390247345\n",
      "184 0.07455314695835114\n",
      "185 0.07195807993412018\n",
      "186 0.0694572925567627\n",
      "187 0.06704624742269516\n",
      "188 0.06472212076187134\n",
      "189 0.06248284503817558\n",
      "190 0.06032408028841019\n",
      "191 0.05824636295437813\n",
      "192 0.05624156817793846\n",
      "193 0.05430742725729942\n",
      "194 0.05244322866201401\n",
      "195 0.05064473673701286\n",
      "196 0.04891257360577583\n",
      "197 0.04724060744047165\n",
      "198 0.045627202838659286\n",
      "199 0.04407069832086563\n",
      "200 0.042569514364004135\n",
      "201 0.041121456772089005\n",
      "202 0.03972352296113968\n",
      "203 0.03837466984987259\n",
      "204 0.03707360476255417\n",
      "205 0.03581662476062775\n",
      "206 0.03460357338190079\n",
      "207 0.033433374017477036\n",
      "208 0.03230363130569458\n",
      "209 0.03121386282145977\n",
      "210 0.030161719769239426\n",
      "211 0.029146255925297737\n",
      "212 0.0281656626611948\n",
      "213 0.027219850569963455\n",
      "214 0.02630627527832985\n",
      "215 0.02542365901172161\n",
      "216 0.024571329355239868\n",
      "217 0.023748591542243958\n",
      "218 0.02295507676899433\n",
      "219 0.02218860201537609\n",
      "220 0.021448368206620216\n",
      "221 0.020733477547764778\n",
      "222 0.020042408257722855\n",
      "223 0.01937490701675415\n",
      "224 0.01873050630092621\n",
      "225 0.018108151853084564\n",
      "226 0.0175067950040102\n",
      "227 0.01692628115415573\n",
      "228 0.016365256160497665\n",
      "229 0.015823066234588623\n",
      "230 0.01529981940984726\n",
      "231 0.01479396689683199\n",
      "232 0.014305006712675095\n",
      "233 0.013832983560860157\n",
      "234 0.013376609422266483\n",
      "235 0.012935799546539783\n",
      "236 0.01251040119677782\n",
      "237 0.012098999693989754\n",
      "238 0.011701245792210102\n",
      "239 0.011316933669149876\n",
      "240 0.01094549149274826\n",
      "241 0.010586582124233246\n",
      "242 0.010239629074931145\n",
      "243 0.00990412849932909\n",
      "244 0.009579958394169807\n",
      "245 0.009266593493521214\n",
      "246 0.008964015170931816\n",
      "247 0.008671218529343605\n",
      "248 0.00838819146156311\n",
      "249 0.008114742115139961\n",
      "250 0.007850305177271366\n",
      "251 0.007594636175781488\n",
      "252 0.00734752370044589\n",
      "253 0.007108685094863176\n",
      "254 0.0068777198903262615\n",
      "255 0.006654491648077965\n",
      "256 0.00643847556784749\n",
      "257 0.0062295966781675816\n",
      "258 0.006027619820088148\n",
      "259 0.00583245512098074\n",
      "260 0.005643734708428383\n",
      "261 0.005461179651319981\n",
      "262 0.0052845836617052555\n",
      "263 0.005113768856972456\n",
      "264 0.004948692861944437\n",
      "265 0.004788994323462248\n",
      "266 0.004634654615074396\n",
      "267 0.004485533107072115\n",
      "268 0.0043410006910562515\n",
      "269 0.004201184026896954\n",
      "270 0.0040660277009010315\n",
      "271 0.003935322165489197\n",
      "272 0.003808833658695221\n",
      "273 0.0036864527501165867\n",
      "274 0.0035681037697941065\n",
      "275 0.0034537797328084707\n",
      "276 0.003343166084960103\n",
      "277 0.003236097516492009\n",
      "278 0.0031324184965342283\n",
      "279 0.0030321269296109676\n",
      "280 0.0029351895209401846\n",
      "281 0.002841387642547488\n",
      "282 0.0027505429461598396\n",
      "283 0.002662606304511428\n",
      "284 0.00257746665738523\n",
      "285 0.0024951796513050795\n",
      "286 0.002415494527667761\n",
      "287 0.002338399412110448\n",
      "288 0.0022638023365288973\n",
      "289 0.0021916311234235764\n",
      "290 0.0021217872854322195\n",
      "291 0.002054236363619566\n",
      "292 0.001988844247534871\n",
      "293 0.0019255831139162183\n",
      "294 0.0018643620423972607\n",
      "295 0.001805186620913446\n",
      "296 0.001747872564010322\n",
      "297 0.001692404504865408\n",
      "298 0.001638701418414712\n",
      "299 0.0015867336187511683\n",
      "300 0.0015364259015768766\n",
      "301 0.0014877920038998127\n",
      "302 0.0014409550931304693\n",
      "303 0.00139564776327461\n",
      "304 0.0013517887564375997\n",
      "305 0.0013093347661197186\n",
      "306 0.0012682456290349364\n",
      "307 0.001228468376211822\n",
      "308 0.0011899746023118496\n",
      "309 0.001152687007561326\n",
      "310 0.0011166068725287914\n",
      "311 0.001081673544831574\n",
      "312 0.0010478526819497347\n",
      "313 0.0010151065653190017\n",
      "314 0.0009833975927904248\n",
      "315 0.0009526923531666398\n",
      "316 0.0009229715797118843\n",
      "317 0.0008942109998315573\n",
      "318 0.0008663521730341017\n",
      "319 0.0008393761236220598\n",
      "320 0.0008132547955028713\n",
      "321 0.0007879676413722336\n",
      "322 0.0007634502253495157\n",
      "323 0.0007397126173600554\n",
      "324 0.0007167413714341819\n",
      "325 0.0006944824708625674\n",
      "326 0.0006729182787239552\n",
      "327 0.0006520544993691146\n",
      "328 0.0006318282685242593\n",
      "329 0.0006122314953245223\n",
      "330 0.0005932760541327298\n",
      "331 0.0005748984985984862\n",
      "332 0.0005571090150624514\n",
      "333 0.0005398662178777158\n",
      "334 0.0005231816903688014\n",
      "335 0.0005070012994110584\n",
      "336 0.0004913388984277844\n",
      "337 0.00047614402137696743\n",
      "338 0.00046144722728058696\n",
      "339 0.00044721851008944213\n",
      "340 0.00043342786375433207\n",
      "341 0.0004200678667984903\n",
      "342 0.00040712812915444374\n",
      "343 0.00039457535604014993\n",
      "344 0.0003824233135674149\n",
      "345 0.00037065756623633206\n",
      "346 0.00035926353302784264\n",
      "347 0.0003482107713352889\n",
      "348 0.0003375117084942758\n",
      "349 0.0003271244349889457\n",
      "350 0.0003170808486174792\n",
      "351 0.00030734614119865\n",
      "352 0.00029789935797452927\n",
      "353 0.0002887680020648986\n",
      "354 0.0002799097856041044\n",
      "355 0.00027132174000144005\n",
      "356 0.0002630018279887736\n",
      "357 0.00025494638248346746\n",
      "358 0.00024713834864087403\n",
      "359 0.00023958155361469835\n",
      "360 0.000232245511142537\n",
      "361 0.00022514448210131377\n",
      "362 0.00021825343719683588\n",
      "363 0.00021157725132070482\n",
      "364 0.00020510431204456836\n",
      "365 0.00019884151697624475\n",
      "366 0.00019277054525446147\n",
      "367 0.00018688033742364496\n",
      "368 0.0001811788824852556\n",
      "369 0.00017565599409863353\n",
      "370 0.00017029485024977475\n",
      "371 0.00016510412388015538\n",
      "372 0.00016006817168090492\n",
      "373 0.00015519047155976295\n",
      "374 0.00015046792395878583\n",
      "375 0.00014588504564017057\n",
      "376 0.0001414426660630852\n",
      "377 0.00013713941734749824\n",
      "378 0.0001329602673649788\n",
      "379 0.00012891808000858873\n",
      "380 0.00012499878357630223\n",
      "381 0.00012119506573071703\n",
      "382 0.00011751511920010671\n",
      "383 0.00011394637112971395\n",
      "384 0.00011047930456697941\n",
      "385 0.00010712855873862281\n",
      "386 0.00010387607471784577\n",
      "387 0.00010072300210595131\n",
      "388 9.766496805241331e-05\n",
      "389 9.470189979765564e-05\n",
      "390 9.182625217363238e-05\n",
      "391 8.904034621082246e-05\n",
      "392 8.634033292764798e-05\n",
      "393 8.372854063054547e-05\n",
      "394 8.118935511447489e-05\n",
      "395 7.873281720094383e-05\n",
      "396 7.635061047039926e-05\n",
      "397 7.403630297631025e-05\n",
      "398 7.179851672844961e-05\n",
      "399 6.962564657442272e-05\n",
      "400 6.751606269972399e-05\n",
      "401 6.547598604811355e-05\n",
      "402 6.349586328724399e-05\n",
      "403 6.157803727546707e-05\n",
      "404 5.971667997073382e-05\n",
      "405 5.791410876554437e-05\n",
      "406 5.61678534722887e-05\n",
      "407 5.44723188795615e-05\n",
      "408 5.2825624152319506e-05\n",
      "409 5.1231239922344685e-05\n",
      "410 4.968308348907158e-05\n",
      "411 4.818894012714736e-05\n",
      "412 4.673364310292527e-05\n",
      "413 4.532110324362293e-05\n",
      "414 4.395608266349882e-05\n",
      "415 4.2632647819118574e-05\n",
      "416 4.1350172978127375e-05\n",
      "417 4.010117118014023e-05\n",
      "418 3.8895734178368e-05\n",
      "419 3.7722111301263794e-05\n",
      "420 3.6589546652976424e-05\n",
      "421 3.5487824789015576e-05\n",
      "422 3.442101296968758e-05\n",
      "423 3.3384323614882305e-05\n",
      "424 3.238128192606382e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "425 3.1406325433636084e-05\n",
      "426 3.0463617804343812e-05\n",
      "427 2.954726187454071e-05\n",
      "428 2.865969690901693e-05\n",
      "429 2.7798820156021975e-05\n",
      "430 2.6964484277414158e-05\n",
      "431 2.615593621158041e-05\n",
      "432 2.5366945919813588e-05\n",
      "433 2.460663927195128e-05\n",
      "434 2.3869777578511275e-05\n",
      "435 2.3153837901190855e-05\n",
      "436 2.2457608793047257e-05\n",
      "437 2.178503382310737e-05\n",
      "438 2.113391565217171e-05\n",
      "439 2.0499797756201588e-05\n",
      "440 1.9884930225089192e-05\n",
      "441 1.928942401718814e-05\n",
      "442 1.8710994481807575e-05\n",
      "443 1.8150161849916913e-05\n",
      "444 1.7608286725590006e-05\n",
      "445 1.7079788449336775e-05\n",
      "446 1.6568703358643688e-05\n",
      "447 1.6071722711785696e-05\n",
      "448 1.5590912880725227e-05\n",
      "449 1.5125535355764441e-05\n",
      "450 1.4674023987026885e-05\n",
      "451 1.4234493391995784e-05\n",
      "452 1.3808922631142195e-05\n",
      "453 1.3395805581239983e-05\n",
      "454 1.2994831195101142e-05\n",
      "455 1.2606441487150732e-05\n",
      "456 1.2229614185343962e-05\n",
      "457 1.1865588930959348e-05\n",
      "458 1.1510438525874633e-05\n",
      "459 1.1165910109411925e-05\n",
      "460 1.083310507965507e-05\n",
      "461 1.050994706019992e-05\n",
      "462 1.019635601551272e-05\n",
      "463 9.89215277513722e-06\n",
      "464 9.597328244126402e-06\n",
      "465 9.310896530223545e-06\n",
      "466 9.032687557919417e-06\n",
      "467 8.764443009567913e-06\n",
      "468 8.502686796418857e-06\n",
      "469 8.2485739767435e-06\n",
      "470 8.002756658243015e-06\n",
      "471 7.765212103549857e-06\n",
      "472 7.533889402111527e-06\n",
      "473 7.308329713850981e-06\n",
      "474 7.091407496773172e-06\n",
      "475 6.880168257339392e-06\n",
      "476 6.675149961665738e-06\n",
      "477 6.476412181655178e-06\n",
      "478 6.2840499595040455e-06\n",
      "479 6.097604909882648e-06\n",
      "480 5.9166623032069765e-06\n",
      "481 5.739850621466758e-06\n",
      "482 5.568889264395693e-06\n",
      "483 5.404032435762929e-06\n",
      "484 5.243132363830227e-06\n",
      "485 5.086852979729883e-06\n",
      "486 4.9363757170795e-06\n",
      "487 4.789198555954499e-06\n",
      "488 4.647188234230271e-06\n",
      "489 4.5092697291693185e-06\n",
      "490 4.3745749280788004e-06\n",
      "491 4.24403424403863e-06\n",
      "492 4.1196699385182e-06\n",
      "493 3.996745817858027e-06\n",
      "494 3.878162260662066e-06\n",
      "495 3.763325821637409e-06\n",
      "496 3.6513060877041426e-06\n",
      "497 3.543655793691869e-06\n",
      "498 3.4383285765215987e-06\n",
      "499 3.3369594802934444e-06\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "print(torch.__version__)\n",
    "\n",
    "# N是batch size；D_in是输入大小\n",
    "# H是隐层的大小；D_out是输出大小。\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 创建随机的Tensor作为输入和输出\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# 使用nn包来定义网络。nn.Sequential是一个包含其它模块(Module)的模块。每个Linear模块使用线性函数\n",
    "# 来计算，它会内部创建需要的weight和bias。\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    ")\n",
    "\n",
    "# 常见的损失函数在nn包里也有，不需要我们自己实现\n",
    "loss_fn = torch.nn.MSELoss(size_average=False)\n",
    "\n",
    "learning_rate = 1e-4\n",
    "for t in range(500):\n",
    "    # 前向计算：通过x来计算y。Module对象会重写__call__函数，因此我们可以把它当成函数来调用。\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # 计算loss \n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 梯度清空，调用Sequential对象的zero_grad后所有里面的变量都会清零梯度\n",
    "    model.zero_grad()\n",
    "\n",
    "    # 反向计算梯度。我们通过Module定义的变量都会计算梯度。\n",
    "    loss.backward()\n",
    "\n",
    "    # 更新参数，所有的参数都在model.paramenters()里\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for param in model.parameters():\n",
    "            param -= learning_rate * param.grad"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
